
/*
 * Carrot2 project.
 *
 * Copyright (C) 2002-2011, Dawid Weiss, Stanisław Osiński.
 * All rights reserved.
 *
 * Refer to the full license file "carrot2.LICENSE"
 * in the root folder of the repository checkout or at:
 * http://www.carrot2.org/carrot2.LICENSE
 */

package org.carrot2.examples;

import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;
import java.util.TreeMap;

import org.apache.commons.lang.StringUtils;
import org.carrot2.clustering.lingo.LingoClusteringAlgorithm;
import org.carrot2.core.Cluster;
import org.carrot2.core.Controller;
import org.carrot2.core.ControllerFactory;
import org.carrot2.core.Document;
import org.carrot2.core.ProcessingResult;
import org.carrot2.core.attribute.CommonAttributesDescriptor;
import org.carrot2.examples.clustering.coOccurence;
import org.carrot2.examples.clustering.entity;
import org.tartarus.snowball.ext.PorterStemmer;

import weka.core.Stopwords;
import checkLocation.location;
/**
 * Simple console formatter for dumping {@link ProcessingResult}.
 */
public class ConsoleFormatter
{
	public static StringBuffer sb;
	public boolean postProcessingClustering;
	public boolean postProcessingCapCount;
	public boolean intent;
	public String Query;
	public float postClusterCoverage;
	public boolean reMerge;	//reMerge the postClustered sub clusters
	public HashMap<String,Integer> Location;
	public boolean findLocation;
	public HashMap<String,String> thumbUrl;
	public boolean dispGalleries;
	public location checkLocation;
	public boolean locationQuery;
	public boolean labelStiching;
	public boolean loglookup;
	public boolean debug;
	public ArrayList<coOccurence.entity> gram;
	
	
	public void setPostProcessingCluster(boolean flag)
	{
		postProcessingClustering=flag;
	}
	public void setReMerge(boolean flag)
	{
		reMerge=flag;
	}
	public void setPostClusterCoverage(float param1)
	{
		postClusterCoverage=param1;
	}
	
	public void setLabelStiching(boolean param1){
		labelStiching=param1;
	}
	public void setLogLookup(boolean param1){
		loglookup=param1;
	}
	public void setPostProcessingCapCount(boolean flag)
	{
		postProcessingCapCount=flag;
	}
	
	public void setFindLocation(boolean flag)
	{
		findLocation=flag;
	}
	
	public void setDispGalleries(boolean flag)
	{
		dispGalleries=flag;
	}
	
	public void setDebug(boolean flag)
	{
		debug=flag;
	}
	public void setGram(ArrayList<coOccurence.entity> param1){
		 gram=param1;
		
		
	}
	public ConsoleFormatter(HashMap<String,String> param1){
		
		postProcessingClustering=false;
		postProcessingCapCount=false;
		Query=null;
		postClusterCoverage=0.4f;
		Location=new HashMap<String, Integer>();
		findLocation=false;
		thumbUrl=param1;
		dispGalleries=false;
		checkLocation=new location();
		locationQuery=location.checkLocationQpUs(Query);
		labelStiching=false;
		loglookup=false;
		gram=null;
		debug=false;
	}
	
	
	public ConsoleFormatter(String query,HashMap<String,String> param1)
	{
		postProcessingClustering=false;
		postProcessingCapCount=false;
		Query=query;
		postClusterCoverage=0.4f;
		Location=new HashMap<String, Integer>();
		findLocation=false;
		thumbUrl=param1;
		dispGalleries=false;
		checkLocation=new location();
		if(Query!=null){
		locationQuery=location.checkLocationQpUs(Query);
		}
		//locationQuery=false;
		labelStiching=false;
		loglookup=false;
		gram=null;
		debug=false;
	
	}
	
    public  void displayResults(ProcessingResult processingResult)
    {
        final Collection<Document> documents = processingResult.getDocuments();
        final Collection<Cluster> clusters = processingResult.getClusters();
        final Map<String, Object> attributes = processingResult.getAttributes();

        // Show documents
        if (documents != null)
        {
            displayDocuments(documents);
        }

        // Show clusters
        if (clusters != null)
        {
            displayClusters(clusters);
        }

        // Show attributes other attributes
        displayAttributes(attributes);
    }

    public  void displayDocuments(final Collection<Document> documents)
    {
        System.out.println("Collected " + documents.size() + " documents\n");
        for (final Document document : documents)
        {
            displayDocument(0, document,0);
        }
    }

    public static void displayAttributes(final Map<String, Object> attributes)
    {
        System.out.println("Attributes:");

        String DOCUMENTS_ATTRIBUTE = CommonAttributesDescriptor.Keys.DOCUMENTS;
        String CLUSTERS_ATTRIBUTE = CommonAttributesDescriptor.Keys.CLUSTERS;
        for (final Map.Entry<String, Object> attribute : attributes.entrySet())
        {
            if (!DOCUMENTS_ATTRIBUTE.equals(attribute.getKey())
                && !CLUSTERS_ATTRIBUTE.equals(attribute.getKey()))
            {
                System.out.println(attribute.getKey() + ":   " + attribute.getValue());
            }
        }
    }

    public StringBuffer displayClusters(final Collection<Cluster> clusters)
    {
    	sb=new StringBuffer();
        //displayClusters(clusters, Integer.MAX_VALUE);
    	//10 serves as the maximum no of documents to be displayed 
    	displayClusters(clusters, 10);
        return sb;
    }

    public void displayClusters(final Collection<Cluster> clusters,
        int maxNumberOfDocumentsToShow)
    {
        displayClusters(clusters, maxNumberOfDocumentsToShow,
            ClusterDetailsFormatter.INSTANCE);
    }

    public void displayClusters(final Collection<Cluster> clusters,
        int maxNumberOfDocumentsToShow, ClusterDetailsFormatter clusterDetailsFormatter)
    {
        //System.out.println("\n\nCreated " + clusters.size() + " clusters\n");
    	
        //sb.append("\n\n\nCreated " + clusters.size() + " clusters\n");
        int clusterNumber = 1;

    	if(postProcessingClustering)
    	{
    		//2nd level clustering needs to be done on the 1st level clusters formed by clustering
    		
    		if(!reMerge){
    		
    			//re merging of the 2nd level clusters NOT required
    			for (final Cluster cluster : clusters)
        		{
        			//post processing of the 1st level clusters            	        	
        			List<Cluster> temp=postProcessing(cluster);
        			
        			//displaying the 2nd level Clusters
        			displayClusterPostProcessing(0, "" + clusterNumber++, cluster, maxNumberOfDocumentsToShow,
                        clusterDetailsFormatter,temp,cluster.getId());
            	}
    		}
    		else{
    			
    			//reMerging of 2nd level cluster required    
    			
    			List<Cluster> indexClusters=new ArrayList<Cluster>();
    			HashMap<String,Cluster> hashClusterlabel=new HashMap<String,Cluster>();
    			HashMap<String,Cluster> rankBasedOrigClusters=new HashMap<String,Cluster>();
    			
    			
    			
    			double newScore=clusters.size();
    			System.out.println(newScore);
    			
    			
    			for(Cluster cluster : clusters){
    				
    				List<Cluster> level2Clusters=postProcessing(cluster);
    	        	HashMap<String,Integer> docUniq=new HashMap<String,Integer>();
    	        	float r=0;
    				

            		
    				
    				//merge the clusters with the same name 
    				for (Cluster item:level2Clusters){
    					
    					if(r>postClusterCoverage)break;	//to show clusters untill the docs coverage has been atleast 40%
    					hash(item,docUniq);
    					r=(float)docUniq.keySet().size()/cluster.getDocuments().size();
    					
    					if(!rankBasedOrigClusters.containsKey(item.getLabel())){
    						
    						//entry has not been made till now
    						
    						//escalated score addition						
    							//item.setScore(item.getScore()*cluster.getScore());
    						
    						//score=no of documents
    							//item.setScore((double)item.getAllDocuments().size());
    						//score of original cluster
    							//item.setScore(cluster.getScore());
    						
    						//hashClusterlabel.put(item.getLabel(), item);
    						
    						
    						item.setScore(newScore);
    						
    						
    						rankBasedOrigClusters.put(item.getLabel(),item);
    						
    						
    					}
    					else{
    						//entry already exists need to update the score and the document list in the current cluster attached with the hashed value
    						
    						//Cluster obj=hashClusterlabel.get(item.getLabel());
    						//obj.addDocuments(item.getAllDocuments());
    						
    						//Double score=0.0;
    						
    						//simple doc counts
    							//score=(double)obj.getAllDocuments().size();
    							
    						
    						
    						//escalated summation of scores
    							//score=obj.getScore();
    							//score+=item.getScore()*cluster.getScore();
    							
    						//simple adddition of scores
    							//score=obj.getScore();
    							//score=score+item.getScore();
    						
    						//max of scores
    							//score=obj.gerScore();
    							//score=(score>item.getScore())?score:item.getScore();
    						
    						//max of orig cluster scores
    							//score=obj.getScore();
    							//score=(score>cluster.getScore())?score:cluster.getScore();
    						
    						//obj.setScore(score);
    							
    	    				//hashClusterlabel.put(item.getLabel(), obj);
    	    				
    	    				Cluster abc =rankBasedOrigClusters.get(item.getLabel());
    	    				abc.addDocuments(item.getAllDocuments());
    	    				rankBasedOrigClusters.put(item.getLabel(), abc);
    	    				
    					}     						
    				}	
    				newScore--;
    			}
    			
    			
    			
    			System.out.println("Total Cluster"+indexClusters.size());
    			
    			System.out.println("Unique Clusters"+hashClusterlabel.keySet().size());
    			
    			//displayMergedClusters(hashClusterlabel);
    			if(debug)
    				displayMergedClusters2(rankBasedOrigClusters);
    			else
    				if(!dispGalleries)
    					displayMergedClusters1(rankBasedOrigClusters);
    				else //display Galleries on
    					displayGalleries(rankBasedOrigClusters);
    			
    			
    				
    				
    			
    			
    			
    		}
    		
        }
        	
        else if(postProcessingCapCount) 
        {
        	for (final Cluster cluster : clusters)
            {
        	
        		String label=sortHashMap(postProcessingCountCapentities(cluster));        	
        		displayCluster(0, "" + clusterNumber++, cluster, maxNumberOfDocumentsToShow,
                        clusterDetailsFormatter, label);
               }
        }
        	
        else
        {
        	
        	
        	
        	for (final Cluster cluster : clusters)
               {
        			
        			
        			
        			displayCluster(0, "" + clusterNumber++, cluster, maxNumberOfDocumentsToShow,
        					clusterDetailsFormatter,null);
               }
        }
            
      }
    
    
    /******** Displaying Merged Clusters on the basis of ranking them again***********/    
    public void displayMergedClusters(HashMap<String,Cluster> param1){
    	
    	int clusterNumber=1;
    	List<Cluster> clusterList=new ArrayList<Cluster>();
    	int maxNumberOfDocumentsToShow=10;
    	
    	for(String key:param1.keySet()){
    		clusterList.add(param1.get(key));
    	}
    	
    	Collections.sort(clusterList,Cluster.byReversedWeightedScoreAndSizeComparator(0d));
    	
    	 
		    
		    Iterator itr=clusterList.iterator();
		    
		    while(itr.hasNext()){
		    	
		    	Cluster obj1=(Cluster)itr.next();
		    	Cluster clnew=removeSameDocs(obj1);
		    	if(clnew.getAllDocuments().size()>6)
		    		Location.put(obj1.getLabel(), clnew.getAllDocuments().size());
		    	
		    	displayCluster(0, "" + clusterNumber++,clnew , maxNumberOfDocumentsToShow,
		    			ClusterDetailsFormatter.INSTANCE,obj1.getLabel());		    	
		    	
		    }
		    
		    
		   
		    
    }
    
    /******** Displaying Merged Clusters only on the basis of what their ranking was intitaially Score has been appended to every cluster ***********/
    
    public void displayMergedClusters1(HashMap<String,Cluster> param1){
    	
    	System.out.println("new ranking");
    	int clusterNumber=1;
    	List<Cluster> clusterList=new ArrayList<Cluster>();
    	int maxNumberOfDocumentsToShow=10;
    	
    	for(String key:param1.keySet()){
    		clusterList.add(param1.get(key));
    	}
    	
    	
    	 Comparator<Cluster> valueComparator = new Comparator<Cluster>() {
		        @Override public int compare(Cluster s1, Cluster s2) {
		        	if(s1.getScore()>s2.getScore())
		        		return 0;
		        	else
		        		return 1;
		            
		        }           
		    };
		    
		    
		    Comparator<Integer> intComparator = new Comparator<Integer>() {
		        @Override public int compare(Integer s1, Integer s2) {
		        	if(s1>s2)
		        		return 0;
		        	else
		        		return 1;
		            
		        }           
		    };
		    
		    Collections.sort(clusterList,valueComparator);
    	 		    
		    Iterator itr=clusterList.iterator();
		    
		   while(itr.hasNext()){
		    	
		    	Cluster obj1=(Cluster)itr.next();
		    	
		    	Cluster clnew=removeSameDocs(obj1);
		    	
		    	
		    	/*coOccurence obj2=new coOccurence();
		    	System.out.println(obj1.getLabel());
    			ArrayList<String> obj3=new ArrayList<String>();
    			for(Document d1: clnew.getAllDocuments()){
    				obj3.add(d1.getTitle());
    			}
    			
    			obj2.coOccuringWordsArrayList(obj3);*/
		    	
		    	if(clnew.getAllDocuments().size()>6)
		    		Location.put(obj1.getLabel(), clnew.getAllDocuments().size());
		    	
		    	
		    	displayCluster(0, "" + clusterNumber++,clnew , maxNumberOfDocumentsToShow,
		    			ClusterDetailsFormatter.INSTANCE,obj1.getLabel());		    	
		    	
		    }
		   
		   
		   
		    
    }

    
    
/******* display merged based upon the gram statistics corresponding to the query*****/     
    public void displayMergedClusters2(HashMap<String,Cluster> param1){
    	
    	
    	
    	
    	System.err.println("reranking");
    	
    	if(gram==null){
    		System.err.println("gram statistics not available");    		
    	}

    	
       	
    	System.out.println("gram statistics ranking");
    	
    	int clusterNumber=1;
    	
    	List<Cluster> clusterList=new ArrayList<Cluster>();
    	
    	int maxNumberOfDocumentsToShow=10;
    	
    	for(String key:param1.keySet()){
    		
    		clusterList.add(param1.get(key));
    	}
    	
    	
    	 Comparator<Cluster> valueComparator = new Comparator<Cluster>() {
		        @Override public int compare(Cluster s1, Cluster s2) {
		        	if(s1.getScore()>s2.getScore())
		        		return 0;
		        	else
		        		return 1;
		            
		        }           
		    };
		    
		    
		    Comparator<Integer> intComparator = new Comparator<Integer>() {
		        @Override public int compare(Integer s1, Integer s2) {
		        	if(s1>s2)
		        		return 0;
		        	else
		        		return 1;
		            
		        }           
		    };
		    
    	 		    
		    Iterator itr=clusterList.iterator();
		    
		    HashMap<String,ArrayList<coOccurence.entity>> gramStatsCluster=new HashMap<String,ArrayList<coOccurence.entity>>();
		    
		   while(itr.hasNext()){
		    	
		    	Cluster obj1=(Cluster)itr.next();
		    	
		    	Cluster clnew=removeSameDocs(obj1);
		    	
		    	ArrayList<String> titles= new ArrayList<String>();
		    	for(Document d:clnew.getAllDocuments()){
		    		titles.add(d.getTitle());		    		
		    	}
		    	
		    	coOccurence temp= new coOccurence();
		    	ArrayList<coOccurence.entity>ll=temp.coOccuringWordsArrayList(titles);
		    	
		    	gramStatsCluster.put(obj1.getLabel(), ll);
		    	
		   }
		   
		   int slotting=10;
		   
		   List<String> reOrder=reorderClustersGramStatisitics(gramStatsCluster).subList(0, 10);
		   
		   
		   
		   
		   
		   for(String tt:reOrder){
			   
			   
			   
			   Cluster obj1=param1.get(tt);
			   
			   Cluster clnew=removeSameDocs(obj1);
			   
			   
		    	displayCluster(0, "" + clusterNumber++,clnew , maxNumberOfDocumentsToShow,
		    			ClusterDetailsFormatter.INSTANCE,obj1.getLabel());
			   
			   
		   }
		   
		    	
		   itr=clusterList.iterator();
		   while(itr.hasNext()){
			   
			   Cluster obj1=(Cluster)itr.next();
			   
			   if(reOrder.contains(obj1.getLabel()))continue;
			   
		    	
		    	Cluster clnew=removeSameDocs(obj1);
		    	if(clnew.getAllDocuments().size()<6)continue;
		    	
		    	if(clnew.getAllDocuments().size()>6)
		    		Location.put(obj1.getLabel(), clnew.getAllDocuments().size());
		    	
		    	displayCluster(0, "" + clusterNumber++,clnew , maxNumberOfDocumentsToShow,
		    			ClusterDetailsFormatter.INSTANCE,obj1.getLabel());
		    	
			   
		   }
				   
		    	
		    	/*
		    	coOccurence obj2=new coOccurence();
		    	System.out.println(obj1.getLabel());
    			ArrayList<String> obj3=new ArrayList<String>();
    			for(Document d1: clnew.getAllDocuments()){
    				obj3.add(d1.getTitle());
    			}
    			
    			obj2.coOccuringWordsArrayList(obj3);
    			*/
		    	
		    /*	if(clnew.getAllDocuments().size()>6)
		    		Location.put(obj1.getLabel(), clnew.getAllDocuments().size());
		    	
		    	
		    	displayCluster(0, "" + clusterNumber++,clnew , maxNumberOfDocumentsToShow,
		    			ClusterDetailsFormatter.INSTANCE,obj1.getLabel());		    	
		    	
		    */
			   
		    
    }

    
    
    public ArrayList<String> reorderClustersGramStatisitics(HashMap<String,ArrayList<coOccurence.entity>> param1){
    	
    	HashMap<String,ArrayList<entity>> invertedIndex=new HashMap<String,ArrayList<entity>>();
    	
    	ArrayList<String> reOrder= new ArrayList<String>();
    	
    	
    	
    	for(String cluster:param1.keySet()){
    		
    		for(coOccurence.entity obj1: param1.get(cluster)){
    			
    			if(!invertedIndex.containsKey(obj1.str))
    				invertedIndex.put(obj1.str, new ArrayList<entity>());
    				
    			
    			entity abc= new entity();
    			abc.str=cluster;
    			abc.value=obj1.val;
    			
    			invertedIndex.get(obj1.str).add(abc);
    				
    			
    		}    		    		
    	}
    	
    	
    	Comparator<entity> intComparator = new Comparator<entity>() {
	        @Override public int compare(entity s1, entity s2) {
	        	if(s1.value>s2.value)
	        		return 0;
	        	else
	        		return 1;
	            
	        }           
	    };
    	
    	
	    System.out.println("size"+gram.size());
	    for(coOccurence.entity obj1: gram){
	    	System.out.print(obj1.str+",");
	    }
	    
	    System.out.println();
    	for(coOccurence.entity obj1: gram){

    		String label=obj1.str;
    		
    		System.out.println(label);
    		
    		if(invertedIndex.keySet().contains(label)){
    			
        		ArrayList<entity> ll=invertedIndex.get(label);
        		
        		
        		
        		Collections.sort(ll,intComparator);
        		
        		/*for(entity abc:ll){
        			System.out.print(abc.str+"  "+abc.value);
        		}*/
        		
        		
        		PorterStemmer stemmer= new PorterStemmer();
        		entity rep=null;
        		boolean flag=false;
        		for(entity abc:ll){
        			StringTokenizer t1= new StringTokenizer(abc.str);
        			
        			while(t1.hasMoreTokens()){
        				String token=t1.nextToken();
        				stemmer.setCurrent(token.toLowerCase());
        				stemmer.stem();
        				String stemmed=stemmer.getCurrent();
        				if(stemmed.compareTo(label)==0)flag=true;
        				
        			}
        			if(flag){
        				rep=abc;
    					break;
    				}
        			
        		}
        		
        		/*if(!flag){
        			
        			for(entity abc:ll){
        				if(abc.str.compareTo("Other Topics")!=0){
        					rep=abc;
        					flag=true;
        					break;
        				}
        			}
        			
        		}*/
        		
        		if(rep!=null){
        			System.out.println("label   "+ rep.str);
        			if(!reOrder.contains(rep.str))reOrder.add(rep.str);
        		}
        		    			
    		}
    		
    		else
    			continue;
    		    		
    		    		    		    		
    	}
    	System.out.println(reOrder);
    	
    	return reOrder;
    	
    	
    	
    	
    }

public void displayGalleries(HashMap<String,Cluster> param1){
    	
    	System.out.println("gallery display");
    	int clusterNumber=1;
    	List<Cluster> clusterList=new ArrayList<Cluster>();
    	int maxNumberOfDocumentsToShow=10;
    	
    	for(String key:param1.keySet()){
    		clusterList.add(param1.get(key));
    	}
    	
    	
    	 Comparator<Cluster> valueComparator = new Comparator<Cluster>() {
		        @Override public int compare(Cluster s1, Cluster s2) {
		        	if(s1.getScore()>s2.getScore())
		        		return 0;
		        	else
		        		return 1;
		            
		        }           
		    };
		    
		    
		    Comparator<Integer> intComparator = new Comparator<Integer>() {
		        @Override public int compare(Integer s1, Integer s2) {
		        	if(s1>s2)
		        		return 0;
		        	else
		        		return 1;
		            
		        }           
		    };
    	 
		    		    
		    HashMap<Integer,ArrayList<Cluster>> galleries=new HashMap<Integer,ArrayList<Cluster>>();
		    
		    Iterator itr=clusterList.iterator();
		    while(itr.hasNext()){
		    	
		    	Cluster obj1=(Cluster)itr.next();
		    	if(galleries.containsKey(obj1.getScore().intValue())){
		    		ArrayList<Cluster> temp=galleries.get(obj1.getScore().intValue());
		    		temp.add(removeSameDocs(obj1));
		    		galleries.put(obj1.getScore().intValue(),temp);		    		
		    	}
		    	
		    	else{		    		
		    		ArrayList<Cluster> temp=new ArrayList<Cluster>();
		    		temp.add(removeSameDocs(obj1));
		    		galleries.put(obj1.getScore().intValue(),temp);
		    	}
		    	
		    }
		    List<Integer> gallery_keys=new ArrayList<Integer>();
		    gallery_keys.addAll(galleries.keySet());
		    
		    Collections.sort(gallery_keys,intComparator);
		    
		    
		   Iterator itrG=gallery_keys.iterator();
		   while(itrG.hasNext()){
			   int abc=(Integer)itrG.next();
			   
			   displayGallery(galleries.get(abc),abc);
			   
		   }
		    
		     
    }


	public void displayGallery(ArrayList<Cluster> param1,int key){
		
		int clusters;
		clusters=param1.size();
		int rows,columns;
		int xcol,xrow;
		if(clusters%2==0)rows=clusters/2;
		else
			rows=(clusters+1)/2;
		columns=2;
		sb.append("<table border=1>");
		for(xrow=0;xrow<rows;xrow++){
			sb.append("<tr>");
			for(xcol=0;xcol<columns;xcol++){
				if((xrow*2+xcol)<param1.size()){
					sb.append("<td>");					
					displayRepresentative(param1.get(xrow*2+xcol),key);
					sb.append("</td>");
				}
				
			}
			sb.append("</tr>");
		}
		sb.append("</table>");
		
	}
    
    
	/**Display one representative image and append the code for the remainign of the images**/
	public void displayRepresentative(Cluster param1,int key){
		Document temp=param1.getAllDocuments().get(0);
		
	
		final String url = temp.getField(Document.CONTENT_URL);
		
        if (StringUtils.isNotBlank(url))
        {
        	String var=thumbUrl.get(url);
        	
        	var=var.replaceAll("http://pix.zenfs.com", "http://d4.yimg.com/sr");
        	
        	sb.append("<a class='fancybox' rel='"+key+"' href='"+var+"'title='"+temp.getField(Document.TITLE)+"'><img  src=\""+url+"\"/></a>");
        }
	
		
				
		appendBlankImages(param1,key);
		
	}
	
	
	/**Append images url without actually displaying them under the same rel id**/
	/** The main benefit being we do not load all images***/
	
	
	public void appendBlankImages(Cluster param1,int key){
		
		List<Document> dList=param1.getAllDocuments();
		Iterator itr=dList.iterator();
		while(itr.hasNext()){
			
			Document d=(Document)itr.next();
			
			final String url = d.getField(Document.CONTENT_URL);
			
	        if (StringUtils.isNotBlank(url))
	        {
	        	String var=thumbUrl.get(url);
	        	
	        	var=var.replaceAll("http://pix.zenfs.com", "http://d4.yimg.com/sr");
	        	
	        	sb.append("<a class='fancybox' rel='"+key+"' href='"+var+"'title='"+d.getField(Document.TITLE)+"'></a>");
	        }
			
		}
		
	}
	
	
	
    
    public Cluster removeSameDocs(Cluster param1){
    	
    	HashMap<String, Document> uniqDocs=new HashMap<String,Document>();
    	
    	List<Document> docsArray=new ArrayList<Document>();
    	
    	for(Document d:param1.getAllDocuments()){
    		if(!uniqDocs.containsKey(d.getField(Document.CONTENT_URL).toString())){
    			uniqDocs.put(d.getField(Document.CONTENT_URL).toString(),d);
    		}    		    		
    	}
    	
    	
    	for(String key:uniqDocs.keySet()){
    		docsArray.add(uniqDocs.get(key));
    	}

    	Cluster obj1=new Cluster();
    	obj1.addDocuments(docsArray);
    	obj1.setScore(param1.getScore());
    	return obj1;
    	
    }
    
    
    private static String sortHashMap(HashMap<String, Integer> param1){
    	

    	StringBuffer out=new StringBuffer();
    	
        ValueComparator bvc =  new ValueComparator(param1);
        TreeMap<String,Integer> sorted_map = new TreeMap(bvc);
        
        sorted_map.putAll(param1);
        
        for (String key : sorted_map.keySet()) {
        	out.append(key + "/"+sorted_map.get(key)+":");            
        }

        
        return out.toString();
        
    }
    
    
    
    
/*
 * recluster again depending upon the titles counting and using some of the clustering technique amongst lingo or stc
 */
    private static List<Cluster> postProcessing(Cluster param1){
    	
    		ArrayList<Document> documents = new ArrayList<Document>();
    		ArrayList<ArrayList<String>> docArray = new ArrayList<ArrayList<String>>();
    	
    		for (final Document document : param1.getDocuments())
        	{
    			ArrayList<String> docDesc = new ArrayList<String>();	
    			docDesc.add(document.getContentUrl());
    			docDesc.add(document.getTitle());
    			docDesc.add(document.getTitle()); 		
    			docArray.add(docDesc);    		
        	}
    		
    		for (ArrayList<String> row : docArray)
    		{
    			//documents.add(new Document(row[1], row[2], row[0]));
    			documents.add(new Document(row.get(2),row.get(1),row.get(0)));            
    		}
    		

    		final Controller controller = ControllerFactory.createSimple();            
        	final ProcessingResult byTopicClusters = controller.process(documents, null,LingoClusteringAlgorithm.class);
        	final List<Cluster> clustersByTopic = byTopicClusters.getClusters();
        
        	return clustersByTopic;
            		
        }
    	
    /*
     * count the number of capentities in the cluster .. 
     */
    private static HashMap<String,Integer> postProcessingCountCapentities(Cluster param1){
    	
    	HashMap<String, Integer> dict=new HashMap<String, Integer>();
    	
    	for(final Document document:param1.getDocuments()){
    		String temp=document.getSummary();
    		StringTokenizer tokensArray = new StringTokenizer(temp);
    		while(tokensArray.hasMoreTokens()){
    			String token=tokensArray.nextToken();
    			if(dict.containsKey(token)){
    				int value=dict.get(token);
    				dict.put(token, value+1);
    			}
    			else
    				dict.put(token,1);
    		}
    		
    		
    		
    	}

		    	
		return dict;
		
    }
    	
    private static String show(HashMap<String,Integer> obj){
    	String ret="";
    	for (Map.Entry<String, Integer> entry : obj.entrySet()) {
    		ret=ret+entry.getKey()+"/"+entry.getValue()+" ";    	    
    	}    
    	return ret;
    }
    
    
    
    public void displayDocument(final int level, Document document,int clusterScore)
    {

    	final String indent = getIndent(level);
        final String url = document.getField(Document.CONTENT_URL);
        if (StringUtils.isNotBlank(url))
        {
        	String var=thumbUrl.get(url);
        	
        	var=var.replaceAll("http://pix.zenfs.com", "http://d4.yimg.com/sr");
        	sb.append("<td><table><tr><td><a class='fancybox' rel='1' href='"+var+"'title='"+document.getField(Document.TITLE)+"'><img  src=\""+url+"\"/></a></td></tr><tr><td><strong>"+document.getField(Document.TITLE)+"</strong></td></tr><tr><td>"+document.getField(Document.SUMMARY)+"</td></tr></table></td>");
        }
    }
    
    
/*
 * to display all the inner documents as well..
 * 
 */
    private void displayCluster(final int level, String tag, Cluster cluster,
        int maxNumberOfDocumentsToShow, ClusterDetailsFormatter clusterDetailsFormatter, String L)
    {
    	boolean locationFlag=false;
    	
    	
    	String label;
    	if(L!=null)
    		label=L;
    	else
    		label=cluster.getLabel();
    	
    	
    	
  
    	
    	
    	
    	
    	

        // indent up to level and display this cluster's description phrase
        for (int i = 0; i < level; i++)
        {
            //System.out.print("  ");
            sb.append("  ");
            
        }
        
    if(labelStiching){
    
       String newLabel=appendLabel(label);
       System.err.println("here"+newLabel);
       label=newLabel;
       
        
    }
        
        if(loglookup){
        	location abc=new location();
        	String ret=abc.labelPredictionQueries(label,this.Query).toLowerCase();
        	StringTokenizer t1= new StringTokenizer(label);
        	boolean flag=true;
        	while(t1.hasMoreTokens()){
        		String t2=t1.nextToken().toLowerCase();
        		if(!Stopwords.isStopword(t2))
        		{
        			if(ret.indexOf(t2)<0){
            			flag=false;
            		}        			
        		}
        		
        		
        	}
        	if(flag)
        	{
        		if(ret.toLowerCase().compareTo(label.toLowerCase())!=0){
        			label+="||"+ret;
        			//label=ret;
        		}
        	}
        	else{
        		
        		//where the suggestuion is not coming from the query logs try appending it with own intellegence
        		String newLabel=appendLabel(label);
        		if(newLabel.compareTo(label)!=0)
        			label+="||"+newLabel;
        			//label=newLabel;
        		
        		
        			
        		
        	}
        }
        
        
        	
        
        
        
        sb.append("<br><a href='http://stage2.images.search.yahoo.com/search/images?p="+label+"&ei=utf-8&y=Search&imgs=yahoohq' target='blank'>"+label + " </a> "+ clusterDetailsFormatter.formatClusterDetails(cluster)+"\n");

        
        
        int documentsShown = 0;        
        //20 has been suggested by Prasenjit 5% of the total document size(400)
        //threshold about which clusters to be shown.. and which not to be shown
        
        if(cluster.getAllDocuments().size()>0){
        	
        	sb.append("<table border=0><tr>");
        	for (final Document document : cluster.getDocuments())
	        {
	            if (documentsShown >= maxNumberOfDocumentsToShow)
	            {
	                break;
	            }
	            displayDocument(level + 1, document,cluster.getScore().intValue());
	            documentsShown++;
	        }
        
	        sb.append("</tr></table>");
        }

        if (maxNumberOfDocumentsToShow > 0
            && (cluster.getDocuments().size() > documentsShown))
        {

            sb.append("\n"+getIndent(level + 1) + "... and "
                    + (cluster.getDocuments().size() - documentsShown) + " more\n");
        }

        // finally, if this cluster has subclusters, descend into recursion.
        final int num = 1;
        for (final Cluster subcluster : cluster.getSubclusters())
        {
            displayCluster(level + 1, tag + "." + num, subcluster,
                maxNumberOfDocumentsToShow, clusterDetailsFormatter,L);
        }
    }
    
    
    
    
    /*
     * to check if the label contains a location such tha we can append the query with the location
     * eg label=> Alberta	Query=>Mountains
     * reformed label=>Mountains in Alberta
     */
    
    public boolean checkLabelForLocation(String label){
    	
    	boolean locationFlag=false;
    	
    	if(checkLocation.checkLocationGWS(label)){
    	//if(checkLocation.checkLocationQpUs(label)){
    		/*
    		 * to stem the phrase and see if the query is already there in the label itself for example in the case of
    		 * Mountain bikers
    		 * Mountain and Reflection
    		 * Mountains in Snow
    		 * 
    		 * The reason is that GWS gives us all these as locations so just a work around
    		 */
    		locationFlag=querySubstringLabel(label);
    	}
    	
    	return locationFlag;
    }
    
    
    /*
     * to decide if the label has to be modified or not
     */
    public boolean querySubstringLabel(String label){
    	

        PorterStemmer obj=new PorterStemmer();
        
    	boolean locationFlag=false;
    	
    	String temp1=label.toLowerCase();
        
        String temp2=Query.toLowerCase();
        
        obj.setCurrent(temp1);
        obj.stem();
        String temp11=obj.getCurrent();
        
        obj.setCurrent(temp2);
        obj.stem();
        String temp21=obj.getCurrent();
        
        
        if(temp11.indexOf(temp21)>=0){
        	
        	locationFlag=false;
        	
        }

        else{
        	
            
            obj.setCurrent(temp2);
            
            obj.stem();
            
            String temp3=obj.getCurrent();
            
            StringTokenizer token=new StringTokenizer(temp1);
            
            
            boolean flag=true;
            
            while(token.hasMoreTokens()){
            	
            	obj.setCurrent(token.nextToken());
            	
                obj.stem();
                
                if(obj.getCurrent().compareTo(temp3)==0){flag=false;}
                
            }
            
            if(flag)
            {
            	locationFlag=true;
            }

        	
        }

        return locationFlag;
    }
    
    
    
    
    private  void displayClusterPostProcessing(final int level, String tag, Cluster cluster,
            int maxNumberOfDocumentsToShow, ClusterDetailsFormatter clusterDetailsFormatter,List<Cluster> temp,int id)
        {
        	String label;
        	StringBuffer labelBuffer=new StringBuffer();
        	HashMap<String,Integer> docUniq=new HashMap<String,Integer>();
        	
        	int count=0;
        	if(temp.size()>2){	//to check if there are some clusters and not just the others cluster which has been created out of the documents.
        		
        		float r=0;;
        		for(Cluster inCluster:temp){
        			if(r>postClusterCoverage)break;	//to show clusters untill the docs coverage has been atleast 40%
        			hash(inCluster,docUniq);
        			r=(float)docUniq.keySet().size()/cluster.getDocuments().size();
        			labelBuffer.append(displaySubCluster(inCluster,id)+"<u><font onClick='display(this.id)' id='"+inCluster.getLabel().replace(' ', '_')+id+"'>"+inCluster.getLabel()+"</font></u>, ");
        			count++;    			
        		}
        		
        	labelBuffer.append(r*100+"%, ");	
        	labelBuffer.append("Total Subcluster::"+temp.size());
        	
            label=labelBuffer.toString();
        	}
        	else
            	label=cluster.getLabel();
        	

            // indent up to level and display this cluster's description phrase
            for (int i = 0; i < level; i++)
            {
                //System.out.print("  ");
                sb.append("  ");
                
            }
            
            sb.append("<br>"+label + "  "+ clusterDetailsFormatter.formatClusterDetails(cluster)+"\n");
            
            
            int documentsShown = 0;        
            //20 has been suggested by Prasenjit 5% of the total document size(400)
            if(cluster.getAllDocuments().size()>0){
            sb.append("<table border=0><tr>");
            for (final Document document : cluster.getDocuments())
            {
                if (documentsShown >= maxNumberOfDocumentsToShow)
                {
                    break;
                }
                displayDocument(level + 1, document,cluster.getScore().intValue());
                documentsShown++;
            }
            
            sb.append("</tr></table>");
            }

            if (maxNumberOfDocumentsToShow > 0
                && (cluster.getDocuments().size() > documentsShown))
            {

                sb.append("\n"+getIndent(level + 1) + "... and "
                        + (cluster.getDocuments().size() - documentsShown) + " more\n");
            }


            final int num = 1;
            for (final Cluster subcluster : cluster.getSubclusters())
            {
                displayClusterPostProcessing(level + 1, tag + "." + num, subcluster,
                    maxNumberOfDocumentsToShow, clusterDetailsFormatter,temp,id);
            }
        }


    /*
     * To return the documents belonging to the cluster as a string of table images
     */
    
    private static String displaySubCluster(Cluster obj,int id){
    	
    	int maxDocs=5;
    	int docShown=0;
    	StringBuffer abc=new StringBuffer();
    	abc.append("<div id=\""+obj.getLabel().replace(' ','_')+id+"\" style='display:none' >");
    	
    	abc.append("<table border='3'><tr><td colspan='3'><strong>"+obj.getLabel()+"-"+obj.getDocuments().size()+"</strong></td></tr><tr>");
    	
        for (final Document document : obj.getDocuments())
        {
            if (docShown >= maxDocs)
            {
                break;
            }
            
            final String url = document.getField(Document.CONTENT_URL);
            if (StringUtils.isNotBlank(url))
            {

            	abc.append("<td><table><tr><td><img src=\""+url+"\"/></td></tr><tr><td><strong>"+document.getField(Document.TITLE)+"</strong></td></tr><tr><td>"+document.getField(Document.SUMMARY)+"</td></tr></table></td>");

            }
            
            docShown++;
        }
        abc.append("</tr></table>");
        abc.append("</div>");
        
        return abc.toString();
    }

    /*
     * To display only the headings for the clusters..
     */
    
    
    private static void displayClusterHeading(final int level, String tag, Cluster cluster,
            int maxNumberOfDocumentsToShow, ClusterDetailsFormatter clusterDetailsFormatter)
        {
            final String label = cluster.getLabel();
            
            for (int i = 0; i < level; i++)
            {           
                sb.append("  ");                
            }
           
            sb.append(label + "\n");           
            int documentsShown = 0;            
            for (final Document document : cluster.getDocuments())
            {
                if (documentsShown >= maxNumberOfDocumentsToShow)
                {
                    break;
                }
                documentsShown++;
            }
            
            if (maxNumberOfDocumentsToShow > 0
                && (cluster.getDocuments().size() > documentsShown))
            {
                //System.out.println(getIndent(level + 1) + "... and "+ (cluster.getDocuments().size() - documentsShown) + " more\n");
                sb.append("\n"+getIndent(level + 1) + "... and "
                        + (cluster.getDocuments().size() - documentsShown) + " more\n");
            }
            final int num = 1;
            for (final Cluster subcluster : cluster.getSubclusters())
            {
                displayClusterHeading(level + 1, tag + "." + num, subcluster,
                    maxNumberOfDocumentsToShow, clusterDetailsFormatter);
            }
        }
    
    private static String getIndent(final int level)
    {
        final StringBuilder indent = new StringBuilder();
        for (int i = 0; i < level; i++)
        {
            indent.append("  ");
        }

        return indent.toString();
    }

    public static class ClusterDetailsFormatter
    {
        public final static ClusterDetailsFormatter INSTANCE = new ClusterDetailsFormatter();

        protected NumberFormat numberFormat;

        public ClusterDetailsFormatter()
        {
            numberFormat = NumberFormat.getInstance();
            numberFormat.setMaximumFractionDigits(2);
        }

        public String formatClusterDetails(Cluster cluster)
        {
            final Double score = cluster.getScore();
            return "(" + cluster.getAllDocuments().size() + " docs"
                + (score != null ? ", score: " + numberFormat.format(score) : "") + ")";
        }
    }
    public static void hash(Cluster inCluster, HashMap<String,Integer> abc){
    	for(Document doc:inCluster.getDocuments()){
    		
    		abc.put(doc.getField(Document.CONTENT_URL).toString(),1);
    		
    	}
    }
    
    public String appendLabel(String label){
    	
    	
    	String ret="";
    	
    	if(!locationQuery)
        {
        //Query is not a location based query
        	if(checkLabelForLocation(label)){
            	
            	ret=location.firstCapital(Query)+" in "+label;
            	System.err.println(ret);
            }
        	else
        		ret=label;
        	
        }
       else{
        	
        	if(querySubstringLabel(label)){
        		
        		ret=label+"  "+location.firstCapital(Query);
        		System.err.println(ret);
        	}
        		
        	else
        		ret=label;
        }
    	
    	return ret;
    	
    }
    
}
